/* 
 * @Author: jade
 * @Date:   2015-12-21 10:33:08
 * @Last Modified by:   jade
 * @Last Modified time: 2016-01-14 19:30:45
 */
var Crawler = require("crawler");
var fs = require('fs');
var path = require('path');
var argv = require('yargs').argv;
var city = argv.city;
//node lianjia.js --city sh
//sh:上海，
var rootPath = 'http://' + city + '.lianjia.com';
var page = 1;

var today = new Date().getDate();

function getDate() {
    var d = new Date();
    return d.getFullYear() + '-' + (d.getMonth() + 1) + '-' + d.getDate();
}

function store(str, fileName) {
    fs.appendFile(fileName || './xiaoqu_' + city + '.txt', str + '\n', function(err) {
        err && console.log(err)
    });
}


function getRandom() {
    return ~~(Math.random() * 10) + 0.1
}

if (page == 1) {
    var tilte = '区域,板块,小区名,小区地址'
    store(tilte);
}

function addQ(url, timer, cb, text) {
    (function(url, timer, cb, text) {
        setTimeout(function() {
            console.log(text);
            c.queue(url);
            cb && cb();
        }, timer)
    })(url, timer, cb, text)
}

var c = new Crawler({
    maxConnections: 300,
    //skipDuplicates: true,
    // This will be called for each crawled page
    callback: function(error, result, $) {
        if(error) {
            console.log('-----------出错了：');
            console.log(error);
            return false;
        }
        if(typeof  $ != 'function') {
            console.log('-----------出错了1：');
            console.log(result);
            return false;
        }
        var url = result.uri;

        if (url.indexOf(rootPath + '/sitemap')) {
            if (url.indexOf('?xiaoqu') > 0) {
                //小区列表页面，先拿到总页数
                var links = $('.house-lst-page-box');
                var pages = 1;

                try {
                    pages = JSON.parse(links.attr('page-data')).totalPage;
                } catch (e) {

                }
                var quyu = $('.option-list .on').first().text();
                var bankuai = $($('.option-list .on')[1]).text();
                for (var i = 0; i < pages; i++) {
                    var urlS = url.split('/');

                    var id = urlS[urlS.length - 2]
                    var nurl = rootPath + '/xiaoqu/' + id + '/pg' + (i + 1) + '/?liebiao';
                    var timer = getRandom() * 1000 * 60 * 1 * (i + 1);
                    //var timer = getRandom() * 1000 * (i + 1);
                    var logStr = '222222:' + timer / 1000 + 's后开始爬第' + (i + 1) + '页小区：'
                    console.log(logStr);
                    addQ(nurl, timer, function() {}, logStr)
                }
            } else {
                if (url.indexOf('?liebiao') > 0) {
                    //小区列表
                    $('#house-lst li .info-panel h2 a').each(function(itemIndex, item) {
                        var href = $(item).attr('href');

                        var nurl = rootPath + href;
                        console.log(nurl)
                        var timer = getRandom() * 1000 * 30 * (itemIndex + 1);
                        //var timer = getRandom() * 1000 * (itemIndex + 1);
                        var logStr = '333333:' + timer / 1000 + 's后开始爬第' + ((page - 1) * 30 + itemIndex + 1) + '条小区：';
                        console.log(logStr);
                        addQ(nurl, timer, function() {}, logStr)
                    })
                } else {
                    //小区详情
                    var str = [];
                    str.push($($('.intro a')[2]).text().replace('小区',''));
                    str.push($($('.intro a')[3]).text().replace('小区',''));
                    str.push($('.res-top h1').text());
                    str.push($('.res-top .adr').text());
                    console.log('-----------' + str.join(','))
                    store(str.join(','));
                }
            }

        } else {
            //sitemap
            $('.div_con').first().find('.list dd a').each(function(itemIndex, item) {
                var timer = 1000 * 60 * 5 * (itemIndex + 1);
                //var timer = getRandom() * 100 * (itemIndex + 1);
                var href = $(item).attr('href');
                var urlS = href.split('/');
                var id = urlS[urlS.length - 2]
                var url = rootPath + '/xiaoqu/' + id + '/?xiaoqu';
                if (itemIndex == 0) {
                    timer = 1000;
                }
                var logStr = '111111:' + timer / 1000 + 's后开始爬' + $(item).text() + '小区：'
                console.log(logStr);
                addQ(url, timer, function() {}, logStr)
            })
        }
    }
});
//'/xiaoqu/pg1/'

c.queue(rootPath + '/sitemap/')

//c.queue('http://www.iwjw.com/sale/shanghai/');
/*setInterval(function(){
    c.queue('http://www.iwjw.com/seeHouseList/')
}, 5*1000)*/
